#-*-coding:utf-8-*-
'''
Created on 2009/6/17

@author: Hung-Hsi Chen
'''
import urllib, urllib2, string, threading, time, Queue, sys
import fetchstockid

class thread_fetchTWSEStockPrice(threading.Thread):
    stockid = fetchstockid.fetchTWSEStockID()
    count = 0
    #stockid = (1000, 1001)
    
    def __init__(self):
        threading.Thread.__init__(self)
        self.startyear = 1990
        self.endyear = 2008
        self.startmonth = 1
        self.endmonth = 12
        self.lock = threading.Lock()
        self.numofdata = (self.endyear-self.startyear+1)*   \
        (self.endmonth-self.startmonth + 1) * len(thread_fetchTWSEStockPrice.stockid)
        
    def run(self):
        for id in thread_fetchTWSEStockPrice.stockid:
            for year in xrange(self.startyear, self.endyear+1):
                for month in xrange(self.startmonth, self.endmonth+1):
                    formatedMon = string.zfill(month, 2)
                    try:
                        self.lock.acquire()
                        url = "http://www.twse.com.tw/ch/trading/exchange/STOCK_DAY/genpage/Report%s%s/%s%s_F3_1_8_%s.php?STK_NO=%s&myear=%s&mmon=%s"\
                        %(year,  formatedMon, year,  formatedMon,id, id, year,  formatedMon)
                        thread_fetchTWSEStockPrice.count += 1
                        self.lock.release()
    
#                        response = urllib2.urlopen(url)
#                        content = response.read()
#                        filename = "/media/disk-1/stock_data/%s_%s%s.htm"%(id, year, formatedMon)
#                        file = open(filename, 'w')
#                        file.write(content)
#                        file.close()
                        
                        sys.stdout.flush()
                        print "[%.2f] %s: %s complete"%(float(thread_fetchTWSEStockPrice.count)/self.numofdata, self.getName(), filename)
                      
                    except:
                        break
#############################################################
def lockversion_main():
    threadpool = []
    WORKER = 100
#    t1 = time.clock()
    for i in range(WORKER):
        tid = thread_fetchTWSEStockPrice()
        tid.start()
        threadpool.append(tid)
    
#    for i in threadpool:
#        i.join()
#        
#    t2 = time.clock()
#    print "total time = %.3f secs"%(t2-t1)
#############################################################
numoftotaldata = (2008-1990+1)*12*len(fetchstockid.fetchTWSEStockID())
#############################################################
class QueueVersion(threading.Thread):
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self.queue = queue
        
    def run(self):
        while not self.queue.empty():
            urlobj = self.queue.get()
            try:
#                print "%s access %s"%(self.getName(), url)
#                response = urllib2.urlopen(urlobj.url)
#                content = response.read()
#                #print url
#                filename = "/media/disk-1/stock_data/%s_%s%s.htm"%(urlobj.id, urlobj.year, urlobj.month)
#                file = open(filename, 'w')
#                file.write(content)
#                file.close()
                self.queue.task_done()
                print "[%.3f]%s %s complete"%(float(numoftotaldata-self.queue.qsize())/numoftotaldata,self.getName(), urlobj.url)
            except Queue.Empty:
                print "queue is empty"
                exit

#############################################################
class Urlobj:
    def __init__(self, id ,year, month, url):
        self.id = id
        self.year = year
        self.month = month
        self.url = url
        
#############################################################

def fillWithQueue(queue):
    stockid = fetchstockid.fetchTWSEStockID()
    startyear = 1990 
    endyear = 2008
    startmonth = 1
    endmonth = 12
    for id in stockid:
           for year in xrange(startyear,endyear+1):
               for month in xrange(startmonth, endmonth+1):
                   formatedMon = string.zfill(month, 2)
                   url = "http://www.twse.com.tw/ch/trading/exchange/STOCK_DAY/genpage/Report%s%s/%s%s_F3_1_8_%s.php?STK_NO=%s&myear=%s&mmon=%s"\
                   %(year,  formatedMon, year,  formatedMon,id, id, year,  formatedMon)
                   queue.put(Urlobj(id, year, formatedMon, url))
    print "fillwith queue complete"
#############################################################

def queuever_main():
    urlqueue = Queue.Queue(0)
    fillWithQueue(urlqueue)
    WORKER = 100
    
    print "queue main start:"
    t1 = time.clock()
    for i in range(WORKER):
        tid = QueueVersion(urlqueue)
        tid.start()
        
    urlqueue.join()

    t2 = time.clock()
    print "total time = %.3f secs"%(t2-t1)
    return
#############################################################
if __name__ == '__main__':
    queuever_main()
#    lockversion_main()
   
